from pyspark.sql import SparkSession
from pyspark.sql import Row

spark = SparkSession.builder.appName('appName').master('local').config("spark.local.dir", "Y:/SparkTemp/").getOrCreate()
sc = spark.sparkContext

# Load a text file and convert each line to a Row.
lines = sc.textFile("J:/spark-2.2.0-bin-hadoop2.7/examples/src/main/resources/people.txt")
print(lines.count())
print(lines.collect())
print("-------------------")
newLine = sc.textFile("J:/spark-2.2.0-bin-hadoop2.7/examples/src/main/resources/people-m.txt")
print(newLine.count())
print(newLine.collect())
print("-------------------")
eLine = sc.textFile("J:/dmData/ad/contest_dataset_user_profile/*.txt")
print(eLine.count())
#print(eLine.collect())
print("-------------------")
parts = lines.map(lambda l: l.split(","))
people = parts.map(lambda p: Row(name=p[0], age=int(p[1])))

# Infer the schema, and register the DataFrame as a table.
schemaPeople = spark.createDataFrame(people)
schemaPeople.createOrReplaceTempView("people")

# SQL can be run over DataFrames that have been registered as a table.
teenagers = spark.sql("SELECT name FROM people WHERE age >= 13 AND age <= 19")

# The results of SQL queries are Dataframe objects.
# rdd returns the content as an :class:`pyspark.RDD` of :class:`Row`.
teenNames = teenagers.rdd.map(lambda p: "Name: " + p.name).collect()
for name in teenNames:
    print(name)
# Name: Justin